
"""
This fetches random page names within the living people subcategory
from erwin85's toolserver and saves them in the wikinbitDb.
"""
# $Id: get_living_people.py 21 2011-07-30 18:57:11Z postshift@gmail.com $

import re
import sys
import json
import urllib
import traceback
import wikinbitDb
from time import time, sleep
from random import random
from subprocess import Popen, PIPE

from optparse import OptionParser

parser = OptionParser(usage="")
parser.add_option("--num_names", dest="desired_num_names", type=int)
(options, args) = parser.parse_args()

# we will use this DB cursor many times below --- commiting each time
cur = wikinbitDb.conn.cursor()
num_names = 0
while num_names < options.desired_num_names:
    # give it a rest of average one second...
    #sleep(random() * 2)
    try:
        output = Popen(
            ['wget', '--max-redirect=0', '-O -', 'http://toolserver.org/~erwin85/randomarticle.php?lang=en&family=wikipedia&categories=Living+people&subcats=1&d=2'], 
            stdout=PIPE, stderr=PIPE).communicate()[1]
    except Exception, exc:
        print "# FAILED: " + traceback.format_exc(exc)
        continue

    if not output:
        print "# got a blank output from Popen"
        continue

    try:
        urlName = re.search(
            ".*title..(?P<name>\S*)", output, re.M).group('name')
        #name = urllib.unquote(re.sub("_", " ", name))
    except Exception, exc:
        print "# FAILED:\n%s\n\t%s" % (traceback.format_exc(exc), output)
        continue

    if not urlName:
        print "# got a blank name"
        continue

    # store the name, commit it.
    try:
        cur.execute("INSERT INTO articles(urlName, revisionsFetched) VALUES (%s, %s)", (urlName, False))
        cur.execute("INSERT INTO categories(urlName, category) VALUES (%s, %s)", (urlName, "Living_people"))
        wikinbitDb.conn.commit()
    except Exception, exc:
        print traceback.format_exc(exc)
        wikinbitDb.reset()
        cur = wikinbitDb.conn.cursor()
        print "Just in case this was an IntegrityError from attempting to insert a duplicate name, we have reset the wikinbitDb connection."
    print "saved %s" % urlName
    sys.stdout.flush()
    num_names += 1

    if num_names % 10 == 0:
        wikinbitDb.reset()
        cur = wikinbitDb.conn.cursor()

wikinbitDb.conn.close()
print "Done!!!"
